###################################################### 
# This is the finnal code to deal all the conditions #
######################################################

#files structure

"""
py_zlz
├── finnal_code.py
├── userfiles
│   ├── numbers.list
│   ├── wechatpage_links.list
│   ├── output.html
│   ├── wechatpage_links_out.list
│   ├── analyse_dict.list
│   ├── input.txt
│   └── ...
├── sogou_pages
│   ├── sg1
│   ├── sg2
│   └── ...
├── wechatpages
│   ├── 184.html
│   ├── 185.html
│   └── ...
├── products
│   ├── product184.html
│   ├── product185.html
│   └── ...
"""

#imports
import urllib3
http = urllib3.PoolManager()

import os

import shutil

import jieba

import re

import pandas


path = "/home/files/py_zlz/" ########CHANGE HERE
path_user = path + "userfiles/"
path_sogou = path+"sogou_pages/"
path_desktop = "/media/psf/Home/Desktop/"

#menu list
menu_main=\
"""
1.Get wechatpage links (By sougu wechat searcher).
2.Curl the wechatpages and rename them.
3.Select key characters.
4.Finnal output.html.
5.Analyse to xlsx.
999.Quit.
"""
menu_get_wechatpage_links_confirm=\
"""
Want to connect to sogou?(y/n)
"""
menu_get_wechatpage_links_0=\
"""
1.Choose a file with numbers in it.
2.Type in the lowerlimit and upperlimit.
"""
menu_get_wechatpage_links_1=\
"""
Type in the filename with numbers in it(Default:numbers.list):
"""
menu_get_wechatpage_links_l=\
"""
Type in the lowerlimit(eg:200):
"""
menu_get_wechatpage_links_u=\
"""
Type in the upperlimit(eg:210):
"""
menu_curl_wechatpages=\
"""
Type in the filename with links in it(Default:wechatpage_links.list):
"""
menu_select_key_characters=\
"""
Type in the dirname with wechatpages in it(Default:wechatpages):
"""

#functions
def get_wechatpage_links_search(numbers):
    for number in numbers:
            link = "https://weixin.sogou.com/weixin?type=2&ie=utf8&query=浙鳌立志班"+str(number)
            http_page = http.request("GET",link)
            data = http_page.data.decode()
            sogou_temp = open(path_sogou+"sg"+str(number)+".html","w")
            sogou_temp.write(data)
            sogou_temp.close()
def deal_sogou_temp():
    sg_list = os.listdir(path_sogou)
    url_list = []
    for sg in sg_list:
        sg_file = open(path_sogou+sg,"r")
        sg_file_content = sg_file.readlines()
        sg_file.close()
        url = "https://weixin.sogou.com"+sg_file_content[181][67:67+294]
        url_list.append(url+"\n")
    link_file = open(path_user+"sogou_links.list","w")
    link_file.writelines(url_list)
    link_file.close()
def get_wechatpage_links():
    state = "y" if os.listdir(path_sogou) == [] else input(menu_get_wechatpage_links_confirm)
    if state == "y":
        state = int(input(menu_get_wechatpage_links_0))
        if state == 1:
            number_filename = input(menu_get_wechatpage_links_1)#Default:numbers.list
            if number_filename == "":
                number_filename = "numbers.list"
            numberfile = open(path_user + number_filename,"r")
            numbers = numberfile.read().split()
            numberfile.close()
        elif state == 2:
            lowerlimit = int(input(menu_get_wechatpage_links_l))
            upperlimit = int(input(menu_get_wechatpage_links_u))
            numbers = range(lowerlimit,upperlimit+1)
        shutil.rmtree(path_sogou)
        os.mkdir(path_sogou)
        get_wechatpage_links_search(numbers)
    deal_sogou_temp()
    print("DONE!")
    return 0

def curl_wechatpages():
    link_filename = input(menu_curl_wechatpages)#Default:wechatpage_links.list
    if link_filename == "":
        link_filename = "wechatpage_links.list"
    linkfile = open(path_user+link_filename,"r")
    link = linkfile.readlines()
    linkfile.close()
    for eachlink in link:
        http_page = http.request("GET",eachlink)
        data = http_page.data.decode()
        name_index = data.find("浙鳌立志班")
        name = data[(name_index+5):(name_index+8)]
        wechatpage_temp = open(path+"wechatpages/"+name+".html","w")
        wechatpage_temp.write(data)
        wechatpage_temp.close()
    print("DONE!")
    return 0

def select_key_characters():
    deal_dir = input(menu_select_key_characters)#Default:wechatpages
    if deal_dir == "":
        deal_dir = "wechatpages"
    real_dir = path + deal_dir
    wechatpage_list = os.listdir(real_dir)
    for eachfile in wechatpage_list:
        wechatpage = open(real_dir+"/"+eachfile,"r")
        wechatpage_content = wechatpage.readlines()
        wechatpage.close()
        product = open(path+"products/product"+eachfile,"w")
        for line in wechatpage_content:
            if "<section" in line:
                product.write(line)
                break
        product.close()
    print("DONE!")
    return 0

def finnal_output(isheader=True):
    products_dir = path +"products/"
    products_list = os.listdir(products_dir)
    products_list.sort()
    output_list = []
    for product in products_list:
        output_file = open(products_dir+product,"r")
        if isheader == True:
            output_list.append("<h1>"+product[7:10]+"</h1>")
        output_list.append(output_file.read())
        output_file.close()
    output = open(path_user+"output.html","w")
    output.writelines(output_list)
    output.close()
    
    print("DONE!")
    return 0 



def select(s,begin,finnal):
    rule = begin+"(.*?)"+finnal
    result_list = re.findall(rule, s)
    for i in range(len(result_list)):
        result_list[i] = begin+result_list[i]+finnal 
    return result_list

def fenci_list(nummark=""):
    text = open(path_user+"input.txt","r").read()
    if nummark.lower() == "del":
        text = re.sub(r'[0-9]+',"",text)
    words_list = jieba.lcut(text)
    deal_list = []
    for i in words_list:
        if i != " " and i != "\n" and nummark.lower() != "del":
            if i[0] in "0123456789":
                i = nummark+i
            deal_list.append(i)
    return deal_list

def freq_output(deal_list):
    analyse_list = open(path_user+"analyse_dict.list","r").read().split()
    isempty = True if analyse_list[0] == "empty" else False
    print("Isempty? "+str(isempty))
    freq = {}
    for i in deal_list:
        if isempty == True or i in analyse_list:
            if i not in freq:
                freq[i]=0
            freq[i] += 1
    output = pandas.DataFrame({"Name":freq.keys(),"Freq":freq.values()})
    output.to_excel(path_desktop+"freq.xlsx")
    output.to_excel(path_user+"freq.xlsx")

def analyse_xlsx():
    freq_output(fenci_list(nummark="[num]"))

#main
if __name__=="__main__":
    state = int(input(menu_main))
    if state == 1:
        get_wechatpage_links()
    elif state == 2:
        curl_wechatpages()
    elif state == 3:
        select_key_characters()
    elif state == 4:
        finnal_output()
    elif state == 5:
        analyse_xlsx()
    elif state == 999:
        quit()
f = open(path_desktop+"out1","w")
outlist = select("".join(fenci_list(nummark="")),begin="我的志向",finnal="谢谢大家")
for i in outlist:
    f.write(i+"\n\n")
f.close()